import time
# from tools import *
from setting import *
from util.data_utils import *


def main():
    while True:
        # 获取队列中的任务
        group_name, url,push_task_id = Q.get()
        if not group_name or not url:
            continue
        
        unique_id = url.split("/")[-2]
        # 以unique_id作为key，采集时间作为value
        last_time_str = R.get(unique_id)
        
        if last_time_str:
            try:
                last_time = float(last_time_str)
            except Exception:
                last_time = 0
            now_time = time.time()
            
            # 如果距离上次采集小于指定时间，则跳过
            if now_time - last_time < MINUTE_GAP * 60:
                continue
            
        print(getCurTime(), f"【{url} 开始】")
        
        data = fetch_data(group_name, unique_id,url)
        data["push_task_id"] = push_task_id
        
        # 保存采集时间到Redis，key为unique_id，value为当前时间戳
        if toDatabase(data): 
            R.set(unique_id, str(time.time()))


if __name__ == "__main__":
    # url  ="https://www.douban.com/group/topic/326622509/?_spm_id=NjE2NTU2OTc"
    url = "https://www.douban.com/group/topic/328878287/?_spm_id=MTg0NDAxMDMx&_i=0232760aUkZGtN,0232792XjFQJ4B"
    print(get_article_data(url))
    # print(json.dumps(get_article_data(url),ensure_ascii=False))
